// ==UserScript==
// @version     0.12: Sorted (and grouped) Scrapers view by site and path.
// @version     0.11: Shortened "Configure" to "cfg" and fixed graphics bugs.
// @version	0.10: Added regexp scraping and the scraper url sharing tab
// @version     0.9: Bugfixed facets and added item type coniguration
// @version	0.8: Moved to the Exhibit 2.0 API and did much refactoring
// @version	0.7: YUI-ext buttons, and other polishing
// @version	0.6: YUI-ext dialog window which doesn't consume the screen
// @version	0.5: YUI-ext tabs with fixed position and nicer styles
// @version	0.4: Configuration now stored in a tidy JSON blob "config".
// @version	0.3: Thickbox replaced with crude YUI tab panel base. Temporarily turned off config restoring.
// @version	0.2: Nicer thickbox css that doesn't break margins of others. Added facet browsing and field removal. Dropped Google sites from default inclusion (gmail broke). Should work without Firebug installed too now.
// @version	0.1: Already works, a little, sort of.
// @name	Exhibit self-injector thickbox
// @namespace	http://code.google.com/p/ecmanaut/
// @description	Adds Exhibit DIY functionality to any web page. Just add XPath.
// @include	http://*
// @exclude	http://mail.google.com/*
// @exclude	http://docs.google.com/*
// ==/UserScript==
// $Id: exhibit-scraper.user.js 689 2008-07-16 09:29:45Z oyasumi $

(function(){

var debugLevel = 0; // for hacking only

if( !(document.contentType||'').match( /^text\/html/i ) )
  return;

var gears, db, fs, server;
function persist( data ) {
  if( !gears ) {
    if( unsafeWindow.GearsFactory )
      gears = new unsafeWindow.GearsFactory;
    else
      return;
    db = gears.create( "beta.database", "1.0" );
    server = gears.create( "beta.localserver", "1.0" );
    fs = server.createStore( "cached" );
    db.open( "scrapings" );
  }
  for( var i = 0, item; item = data.items[i]; i++ ) {
    ;
  }
}

var baseurl = "http://ecmanaut.googlecode.com/svn/trunk/lib/";
var exhibit = "http://static.simile.mit.edu/exhibit/api-2.0/exhibit-api.js";
var yui_ext = baseurl + "YUI-Ext-0.33.js";
var ext_css = baseurl + "css/yui-ext-css-1228.css";
var gfx_root = "http://gfx.ecmanaut.googlepages.com/";

var ui_id = "ExCfg"; // top of our DOM

var panels = {
  scraper: { title:"Scraper", view:"center" },
  exhibit: { title:"Exhibit", view:"center" },
 scrapers: { title:"Scrapers", view:"center" },
   facets: { title:"Facets", view:"east" }
};

var buttons = { "Remove:":remove_property,
		"Add Another Field":add_field,
		"Scrape Page":render_exhibit };

var looks = {
  modal: true,
  width: innerWidth - 100,
  height: innerHeight - 100,
  shadow: true,
  center: {
    autoScroll: true,
    closeOnTab: true,
    tabPosition: "top",
    alwaysShowTabs: true
  },
  east: {
    split: true,
    titlebar: true,
    autoScroll: true,
    collapsible: true,
    initialSize: 200
  }
};

// Index the scraping template configuration by url, minus query portion:
var key = location.host + location.pathname;

var loaded, button, css;
var Ext, getEl, Exhibit;

var dialog = (function() {
  function css_on() { css.disabled = false; show( ui_id ); }
  function css_off() { css.disabled = true; hide( ui_id ); }
  var dialog, layout, showBtn, center, east, exhibit;
  return {
    init : function() {
      button.href = "#";
      button.innerHTML = "[cfg]";
      this.showBtn = showBtn = getEl( button );
      this.showBtn.on( "click", this.show, this, true );
      var ui = inject_ui();

      this.dialog = dialog = new Ext.LayoutDialog( ui_id, looks );
      dialog.addKeyListener( 27, this.hide, this );

      layout = dialog.getLayout();
      dialog.beginUpdate();
      for( var name in panels ) {
        var panel = new Ext.ContentPanel( name, panels[name] );
        layout.add( panels[name].view, panel );
      }
      center = layout.getRegion( "center" );
      east = layout.getRegion( "east" );
      exhibit = center.getPanel( "exhibit" );
      scrapers = center.getPanel( "scrapers" );
      center.showPanel( "scraper" );
      east.hide();
      exhibit.on( "activate", east.show, east, true );
      exhibit.on( "deactivate", east.hide, east, true  );
      scrapers.on( "activate", list_scrapers );
      dialog.on( "beforeshow", css_on );
      dialog.on( "beforehide", function(){ setTimeout( css_off, 1e3 ); } );

      var mybuts = $x( './/div[@class="mybut"]', ui );
      for( var i = 0, but; but = mybuts[i]; i++ )
	mybuts[i] = make_mybut( but );

      dialog.endUpdate();
    }, // init()

    hide : function() {
      dialog.hide();
    },

    focus : function( panel ) {
      center.showPanel( panel );
    },

    show : function( e ) {
      if( e ) { e.preventDefault(); e.stopPropagation(); }
      if( !dialog ) this.init();
      dialog.show( showBtn.dom );
    }
  };
})();

var exposed = unsafeWindow.exposed = {
  dialog:dialog, // for hacking only
  init: init,
  onload: onload,
  read:function() {
    var form = $(id('form'));
    if (form)
      return read_config( form );
  },
  save:function( json ) {
    configs[key] = { scrape:json };
    GM_setValue( "config", configs.toSource() );
  }
};

var configs;
function load_config( key ) {
  function fetch() {
    configs = eval( GM_getValue( "config", "({})" ) );
    eval_config(key);
  }
  log("load_config");
  setTimeout(fetch, 0);
}

function eval_config( key ) {
  if( !key ) return configs;
/*
  var urlcfg = decode_hash();
  if( urlcfg &&
     (urlcfg = decode_json( urlcfg )) )
    if( (urlcfg != encode_json( config )) &&
        confirm( "Drop old scrape config and use URL scrape settings?" ) )
      config = urlcfg;
*/
  var config = decode_hash();
  if( config ) {
    config = decode_json( config );
    var old = configs[key];
    console.log( config, old );
    if( !old || !old.scrape.fields )
      return config;
    if( encode_json( old.scrape ) != encode_json( config.scrape ) &&
        confirm( "Override old " + (old.scrape.fields.length) +
                 " property scrape setup with new "+
                 (config.scrape.fields.length) + " property setup?" ) )
      return config;
    return old;
  }
  return configs[key];
}

var isJSON = /^("(\\.|[^"\\\n\r])*?\x22|[,:{}\[\]0-9.\-+Eaeflnr-u \n\r\t])+?$/;
function decode_json( json ) {
  try {
    if( isJSON.test( json ) )
      return eval( "("+ json + ")" );
  } finally {}
}

function encode_json( data ) {
  var type = typeof data;
  if( type == "object" && data instanceof Array )
    type = "array";
  switch( type ) {
    case "array":  return "["+ data.map( encode_json ).join( "," ) +"]";
    case "string": return '"'+ data.replace( /([\\\x22])/g, "\\$1" ) +'"';
    case "object":
      var pairs = [];
      for( var prop in data )
        pairs.push( encode_json( prop ) +":"+ encode_json( data[prop] ) );
      return "{" + pairs.join(",") + "}";
  }
  return String( data );
}

function save_config( key, form ) {
  log("save_config");
  configs[key] = { scrape:read_config( form ) };
  GM_setValue( "config", configs.toSource() );
}

// var z = 0;
function read_config( form ) {
  function get() {
    //debug(++z+"?", arguments);
    var name = id.apply( this, Array.prototype.slice.call( arguments ) );
    //debug(z+".");
    var node = form.elements.namedItem( name );
    return node && (node.type == "checkbox" ? node.checked : node.value);
  }
  log( "read_config", form );
  var config = { type:get("type"), root:get("root"), rows:get("rows") };
  var fields = [], field = {}, i, j, prop;
  var max = $X( '(.//tr[starts-with(@id,"'+id('row')+'")])[last()]', form );
  max = no( max.id );
  log( max );
n:for( i = 0; i <= max; i++ ) {
    for( j = 0; prop = scrape_props[j]; j++ ) {
      field[prop] = get( prop, i );
      if( null == field[prop] )
	continue n;
      if( field[prop] == "" )
        delete field[prop];
    }
    fields.push( field );
    field = {};
  }
  config.fields = fields;
  log( "=>", config );
  return config;
}

function decode_hash() {
  var hash = decodeURIComponent( (location.hash || '#').substring( 1 ) );
  if( hash )
    return "{"+ hash +"}";
}

function inject_ui() {
  log("inject_ui");
  var field;
  var config = load_config( key ); // load scraper config from a prior visit
  var scrape = config ? config.scrape : { // or pick a cruddy default template:
    type:'Item',
    root:'/',
    rows:'div',
    fields:[{ name:"label", path:"", regexp:"" }]
  };

  // The config interface HTML bits:
  inject_after( <div id={ui_id}>
<div class="ydlg-hd">Exhibit scraper</div>
<div class="ydlg-bd">
  <div class="ydlg-tab" title="Scraper" id="scraper">

<form id={id('form')} style="padding:6px;"><table cellspacing="4">
<tr>
  <th colspan="2"><label for={id('type')}>Item type name:</label></th>
  <td>
    <input type="text" class="x" style="width:335px;"
	   title="The type items scraped by this scraper are instances of"
	   value={scrape.type||"Item"} id={id('type')}/>
  </td>
  <td id={id('type','match')}/>
</tr>

<tr>
  <th colspan="2"><label for={id('root')}>Scrape root node:</label></th>
  <td>
    <input type="text" class="x" style="width:335px;"
	   title="XPath to scrape root context node"
	   value={scrape.root} id={id('root')}/>
  </td>
  <td id={id('root','match')}/>
</tr>

<tr>
  <th colspan="2"><label for={id('rows')}>Row selector:</label></th>
  <td>
    <input type="text" class="x" style="width:335px;" title="XPath to row"
	   value={scrape.rows} id={id('rows')}/>
  </td>
  <td id={id('rows','match')}/>
</tr>

<tr id={id('row',0)}><th>Fields:</th>
  <th>
    <input type="text" class="n" size="10" title="Attribute name"
	   value="label" id={id('name',0)} readonly="readonly"/>:
  </th>
  <td>
    <input type="text" class="x" size="30" title="XPath to value"
	   value={scrape.fields[0].path} id={id('path',0)}/>
    <select type="text" id={id('type',0)} title="Value Type">
      <option value="text">Text</option>
      <option value="date">Date</option>
      <option value="number">Number</option>
      <option value="boolean">Boolean</option>
      <option value="item">Item</option>
      <option value="url">URL</option>
      <option value="html">HTML</option>
    </select>
    <input type="checkbox" id={id('facet',0)} title="Show as facet"/>
    <input type="text" class="x" size="30"
	   title="RegExp that extracts the value (match group 1)"
	   value={scrape.fields[0].regexp||""} id={id('regexp',0)}/>
  </td>
  <td id={id('match',0)}/>
</tr>

<tr id={id('add-before')}><th colspan="3">
  <div class="mybut" title="Add Another Field"/>
  <div class="mybut" title="Scrape Page" style="float:right;"/>
</th></tr></table></form>

  </div><!-- ydlg-tab (Scraper) -->

  <div class="ydlg-tab" title="Exhibit" id="exhibit">
    <div role="viewPanel" id="exhibit-view-panel"/>
  </div><!-- ydlg-tab (Exhibit) -->

  <div class="ydlg-tab" title="Scrapers" id="scrapers">
  </div><!-- ydlg-tab (Scrapers) -->

  <div class="ylayout-inactive-content" title="Facets" id="facets">
    <div id="exhibit-browse-panel"/>
  </div><!-- ylayout-inactive-content (Facets) -->
</div><!-- ydlg-bd -->
<!-- ExCfg --></div> );
  var ui = $( ui_id );
  ui.innerHTML = ui.innerHTML; // hack: activate event handlers
  list_scrapers();
  for( var i=1; field = scrape.fields[i]; i++ ) {
    add_field( field );
  }
  scrape_page( $(id('form')) );
  return ui;
}

function list_scrapers() {
  var scrapers = load_config();
  var dl = <dl/>;

  var all = {}, sites = [];
  for (var s in scrapers) {
    var m = /^(.*?)(\/.*)/.exec( s );
    var site = m[1], path = m[2];
    if (!all[site]) {
      all[site] = [];
      sites.push( site );
    }
    all[site].push( path );
  }
  sites.sort();

  var dt;
  for (var i = 0; site = sites[i]; i++) {
    if (dt)
      dt += <dt>{site}</dt>;
    else
      dt = <dt>{site}</dt>;
    var dd = <dd/>;
    var ul = <ul/>;
    var paths = all[site].sort();
    for (var j = 0; path = paths[j]; j++) {
      var url = site + path;
      var json = encode_json( scrapers[url] ).replace( /^\{|\}$/g, "" );
      ul.li += <li><a href={"http://"+ url +"#"+ json}>{path}</a></li>;
    }
    dd.ul += ul;
    dt += dd;
  }
  dl.dt += dt;

  var list = $( "scrapers" );
  list.innerHTML = "";
  if (dt)
    inject_after( dl, list );
  list.innerHTML = list.innerHTML; // hack: activate event handlers
}

invoke_script();

function invoke_script() {
  //log('invoke_script');
  if(innerHeight < 100 || innerWidth < 100) return;
  var url = 'javascript:exposed.init()', config_id = 'exhibit-configure-link';
  inject_after( <a href={url} id={config_id} accesskey="x" title="Exhibit configuration" style="position:fixed;z-index:1;bottom:0px;right:0px;font-size:12pt;padding:5px;background-color:#333;opacity:0.8;color:black;text-decoration:none;">[cfg]</a> );
  button = $( config_id );
}

function add_css( css ) {
  var id = 'exhibit-autoinjector';
  if( css instanceof Array ) {
    id += '-private';
    var content = '', root = '#'+ ui_id +' ';
    for( var i = 0, data; data = css[i]; i++ ) {
      var selectors = root + data[0].split(/\s*,\s*/).join(', '+root);
      content += selectors + ' {'+ data[1] +'}\n';
    }
    css = content;
  }
  var url = css.match( /^http/i );
  var style = $( id );
  if( !style ) {
    style = document.createElement( url ? 'link' : 'style' );
    if( url ) style.rel = 'stylesheet';
    style.type = 'text/css';
    style.id = id;
  }
  if( url )
    style.href = css;
  else
    style.innerHTML = css;
  if( !style.parentNode )
    $X( '//head' ).appendChild( style );
  return style;
}

function init() {
  log('init');
  if( loaded ) {
    exposed.dialog.show();
  } else {
    button.innerHTML = '[<i>Loading...</i>]';
    add_script( exhibit + "?js="+ yui_ext +"&css="+ ext_css +
		"&autoCreate=false&callback=exposed.onload" );
    css = add_css([
      ['.exhibit-facet-body','height:auto;'],
      ['.mybut','float:left; margin-right:3px;'],
      ['input,th,.ybtn-center','color:black; background-color:white;'],
      ['input[readonly]','border-style:groove; background:#e3e3e3;'],
      ['.ylayout-panel-hd-text','font-weight:bold; color:#083772;'],
      ['.ylayout-collapsed-east','background:url("'+ gfx_root +
       'right-facets.png") no-repeat center;'],
      ['#scrapers dl','margin:0.5em;'],
      ['#scrapers dd','margin:0 1em 0.5em;'],
      ['#scrapers dt','font-weight:bold;']
     ]);
  }
}

function onload() {
  log('onload');
  Ext = unsafeWindow.YAHOO.ext;
  getEl = unsafeWindow.getEl;
  loaded = 1;
  init();
}

function remove_property() {
  remove_node( this.parentNode.parentNode );
}

function remove_node( node ) {
  node.parentNode.removeChild( node );
}

function render_exhibit() {
  //enter('render_exhibit');
  var form = $( id('form') );
  Exhibit = unsafeWindow.Exhibit;
  var data = scrape_page( form );
  log( "Scraped data: %o", data );
  save_config( key, form );
//  SimileAjax.History.historyFile = "...wherever/we/want/it/to/be.html";
  unsafeWindow.SimileAjax.History.enabled = false;
  exposed.data = data;
  var db = exposed.database = Exhibit.Database.create();
  unsafeWindow.database = db; // FIXME: kludge; why does Exhibit need this?
  db.loadData( data, Exhibit.Persistence.getBaseURL(location.href) );
  exposed.exhibit = Exhibit.create( db );
  exposed.exhibit.configureFromDOM( $("ExCfg") );
  dialog.focus('exhibit');
  //leave('render_exhibit');
}

function scrape_page( form ) {
  function decode( value ) {
    if (typeof value == "object") { // some DOM node
      if (type == "html")
        value = value.innerHTML;
      else
        value = value.nodeValue || value.textContent;
    } else
      value = value;
    if (!regexp && type == "number")
      regexp = /([+-]?\d*[\d.eE+-]*\d)/;
    if (typeof value == "string") {
      if (regexp) {
        var match = regexp.exec( value );
        if (match)
          value = match[1];
      }
      if (type == "text")
        value = value.replace( /^\s*|\s*$/g, "" );
    }
    return value;
  }
  enter( 'scrape_page' );
  var config = exposed.scraper = read_config( form );
  var root = $X( config.root, null, 'root' );
  var rows = $x( config.rows, root, 'rows' );
  var fields = config.fields, props = {};
  var parsed = { items:[] }, facets = "";
  for( var r = 0, row; row = rows[r]; r++ ) {
    // log( 'row ', r, ': ', row );
    var item = { type:config.type }, field, lastvalue;
    for( var c = 0; field = fields[c]; c++ ) {
      var name = field.name, type = field.type, facet = field.facet;
      var value = field.path ? $x( field.path, row, row.id ) : lastvalue;
      lastvalue = value;
      var regexp = field.regexp && new RegExp( field.regexp );
      // log( name, field.path, value, facet );
      if( r == 0 ) { // first outer loop/scraped item: setup facets
        if( facet )
          facets += "<div ex:role='facet' ex:expression='."+ name +
            "' ex:facetLabel='"+ name +"'></div>";
	props[name] = { valueType:field.type };
      }
      if( typeof value == "undefined" ) continue;
      item[name] = (value instanceof Array ? value : [value]).map( decode );
      // log( name, ' [path: ', field.path, ']: ', item[name] );
    }
    parsed.items.push( item );
  }
  parsed.properties = props;

  $( 'exhibit-browse-panel' ).innerHTML = facets;
  leave( 'scrape_page' );
  return parsed;
}

function debug() {
  if( !("console" in unsafeWindow ) ) return;
  var args = Array.prototype.slice.apply( arguments );
  console.debug.apply( this, args );
}

function log() {
  if( !("console" in unsafeWindow ) || !debugLevel ) return;
  var args = Array.prototype.slice.apply( arguments );
  console.log.apply( this, args );
}

function enter( name ) {
  if( !("console" in unsafeWindow ) ) return;
  console.profile( name );
}

function leave( name ) {
  if( !("console" in unsafeWindow ) ) return;
  console.profileEnd( name );
}

var scrape_props = ["name", "path", "regexp", "type", "facet"];

//! Add another row at the end of the config pane.
//! @param values
//!   Optional { name:field-label, path:xpath-expression } specification.
function add_field( values ) {
  log( 'add_field', values );
  if( values && values.scope && values.text ) {
    values = { name:"", path:"", regexp:"" };
    var make = true;
  }
  var trs = $x( 'id("'+ id('form') +'")/table/tbody/tr' );
  var next = trs[trs.length-1];
  var prev = trs[trs.length-2];
  var tr = prev.cloneNode( true );
  var n = no( tr.id ) + 1;
  tr.id = id( 'row', n );

  var parent = tr.getElementsByTagName( 'th' )[0];
  var but = document.createElement( 'div' );
  but.className = 'mybut';
  but.title = 'Remove:';
  parent.innerHTML = '';
  parent.appendChild( but );

  values.type = values.type || 'text';
  var name, prop, node;
  for( var i = 0; prop = scrape_props[i]; i++ ) {
    name = id( prop, '' );
    node = $X( './/*[starts-with(@id,"'+ name + '")]', tr );
    node.id = name + n;
    if( node.type == "checkbox" )
      node.checked = !!values[prop];
    else
      node.value = typeof values[prop] == "undefined" ? "" : values[prop];
    if( prop == 'name' ) {
      node.removeAttribute( 'readonly' );
      node.focus();
    }
  }
  name = id( 'match', '' );
  node = $X( './/*[starts-with(@id,"'+ name + '")]', tr );
  node.id = name + n;
  next.parentNode.insertBefore( tr, next );
  if( make ) make_mybut( but );
}

function make_mybut( but ) {
  var title = but.title;
  var specs = { text:title, handler:buttons[title], scope:but };
  return new Ext.Button(but, specs);
}

function add_script( url ) {
  //var script = <script src={url} type="text/javascript"/>;
  var script = document.createElement( 'script' ); script.src = url;
  var head = $X( '//head[1]' );
  head.appendChild( script );
  //inject_after( script, head );
}

function inject_after( xml, targetNode ) {
  return append_to( xml, targetNode||document.body );
  var node = (new DOMParser).parseFromString( xml.toXMLString(),
					      'text/xml' ).firstChild;
  (targetNode||document.body).appendChild( node );
}

function append_to( e4x, node, doc ) {
  return node.appendChild( import_node( e4x, doc || node.ownerDocument ) );
}

function import_node( e4x, doc ) {
  var me = import_node, xhtml, domTree, import_me;
  me.Const = me.Const || { mimeType: 'text/xml' };
  me.Static = me.Static || {};
  me.Static.parser = me.Static.parser || new DOMParser;
  xhtml = <testing xmlns="http://www.w3.org/1999/xhtml" />;
  xhtml.test = e4x;
  domTree = me.Static.parser.parseFromString( xhtml.toXMLString(),
					      me.Const.mimeType );
  import_me = domTree.documentElement.firstChild;
  while( import_me && import_me.nodeType != 1 )
    import_me = import_me.nextSibling;
  if( !doc ) doc = document;
  return import_me ? doc.importNode( import_me, true ) : null;
}

function show( id ) { $( id ).style.display = 'block'; }
function hide( id ) { $( id ).style.display = 'none'; }

function no( id ) {
  return parseInt( /\d+$/.exec( id )[0], 10 );
}
function id() {
  return [ui_id].concat( Array.prototype.slice.call( arguments ) ).join('-');
}

function $( id ) { return document.getElementById( id ); }

function $X( xpath, root, match, nr ) {
  var got = $x( xpath, root, match, nr );
  return got instanceof Array ? got[0] : got;
}

function $x( xpath, root, match, nr ) {
  try {
  var doc = root ? root.evaluate ? root : root.ownerDocument : document;
  var got = doc.evaluate( xpath, root||doc, null, 0, null ), next, result = [];
  switch( got.resultType ) {
    case got.STRING_TYPE:
      return got.stringValue;
    case got.NUMBER_TYPE:
      return got.numberValue;
    case got.BOOLEAN_TYPE:
      return got.booleanValue;
    default:
      while( next = got.iterateNext() )
	result.push( next );
      return result;
  }
  } catch( e ) {
    var error = e;
  }
}

function node_to_e4x( node ) {
  return new XML( (new XMLSerializer).serializeToString( node ) );
}

function foreach( xpath, cb, root ) {
  var results = $x(xpath, root), node, i;
  for( i = 0; node = results[i]; i++ )
    cb( node, i );
}

})();
